<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2014-2017 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>MapReduce Programs</title>

    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <script type="text/javascript" src="../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../index.html" />
    <link rel="up" title="核心概念" href="index.html" />
    <link rel="next" title="Plugins" href="plugins.html" />
    <link rel="prev" title="Cube Dataset" href="datasets/cube.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: developer-manual -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><a href="../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a>
      </li>
      <li class="toctree-l1"><b><a href="../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../index.html"> 简介</a></li>
<li class="toctree-l1"><a class="reference internal" href="../getting-started/index.html"> 入门指南</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/sandbox/index.html">CDAP Sandbox</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/zip.html">二进制 Zip 文件</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/zip.html#cdap-sandbox">启动和停止 CDAP Sandbox</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/virtual-machine.html">虚拟机镜像</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/docker.html">Docker 镜像</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/quick-start.html">快速入门</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/dev-env.html">搭建开发环境</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/start-stop-cdap.html">启动和停止 CDAP</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/building-apps.html">构建并运行应用</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../overview/index.html"> 概述</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../overview/anatomy.html"> 大数据应用剖析</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/modes.html"> 模式和组件</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/abstractions.html"> 核心概念</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/interfaces.html"> 编程接口</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html"> 抽象概念</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="core.html"> Core Abstractions</a></li>
<li class="toctree-l2"><a class="reference internal" href="applications.html"> Applications</a></li>
<li class="toctree-l2"><a class="reference internal" href="datasets/index.html"> Datasets</a><ul>
<li class="toctree-l3"><a class="reference internal" href="datasets/overview.html"> Overview</a></li>
<li class="toctree-l3"><a class="reference internal" href="datasets/table.html"> Table API</a></li>
<li class="toctree-l3"><a class="reference internal" href="datasets/fileset.html"> FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="datasets/partitioned-fileset.html"> Partitioned FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="datasets/time-partitioned-fileset.html"> TimePartitioned FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="datasets/system-custom.html"> System and Custom Datasets</a></li>
<li class="toctree-l3"><a class="reference internal" href="datasets/permissions.html"> Dataset Permissions</a></li>
<li class="toctree-l3"><a class="reference internal" href="datasets/cube.html"> Cube Dataset</a></li>
</ul>
</li>
<li class="toctree-l2 current"><a class="current reference internal" href="#"> MapReduce Programs</a></li>
<li class="toctree-l2"><a class="reference internal" href="plugins.html"> Plugins</a></li>
<li class="toctree-l2"><a class="reference internal" href="schedules.html"> Schedules</a></li>
<li class="toctree-l2"><a class="reference internal" href="secure-keys.html"> Secure Keys</a></li>
<li class="toctree-l2"><a class="reference internal" href="services.html"> Services</a></li>
<li class="toctree-l2"><a class="reference internal" href="spark-programs.html"> Spark Programs</a></li>
<li class="toctree-l2"><a class="reference internal" href="workers.html"> Workers</a></li>
<li class="toctree-l2"><a class="reference internal" href="workflows.html"> Workflows</a></li>
<li class="toctree-l2"><a class="reference internal" href="artifacts.html"> Artifacts</a></li>
<li class="toctree-l2"><a class="reference internal" href="program-lifecycle.html"> Program Lifecycle</a></li>
<li class="toctree-l2"><a class="reference internal" href="namespaces.html"> Namespaces</a></li>
<li class="toctree-l2"><a class="reference internal" href="transaction-system.html"> Transaction System</a></li>
<li class="toctree-l2"><a class="reference internal" href="transactional-messaging-system.html"> Transactional Messaging System</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../metadata/index.html"> 元数据</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../metadata/system-metadata.html"> System Metadata</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/discovery-lineage.html"> Discovery and Lineage</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/field-lineage.html"> Field Level Lineage</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/audit-logging.html"> Audit Logging</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/metadata-ui.html"> CDAP Metadata UI</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/programmatic-metadata.html"> Accessing metadata programmatically</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../pipelines/index.html"> 数据流管道</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/concepts-design.html"> Concepts and Design</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/getting-started.html"> Getting Started</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/studio.html"> CDAP Studio</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/creating-pipelines.html"> Creating Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/running-pipelines.html"> Running Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/plugin-management.html"> Plugin Management</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/plugins/index.html"> Plugin Reference</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/actions/index.html"> Action Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/sources/index.html"> Source Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/transforms/index.html"> Transform Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/analytics/index.html"> Analytic Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/sinks/index.html"> Sink Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/shared-plugins/index.html"> Shared Plugins</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../pipelines/plugins/shared-plugins/core.html">CoreValidator</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/plugins/post-run-plugins/index.html"> Post-run Plugins</a><ul class="simple">
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/developing-pipelines.html"> Developing Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/developing-plugins/index.html"> Developing Plugins</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/plugin-basics.html">Plugin Basics</a></li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/creating-a-plugin.html">Creating a Plugin</a></li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/presentation-plugins.html">Plugin Presentation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/testing-plugins.html">Testing Plugins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../pipelines/developing-plugins/packaging-plugins.html">Packaging Plugins</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../pipelines/how-cdap-pipelines-work.html"> How CDAP Pipelines Work</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../cloud-runtimes/index.html"> 云平台运行</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/concepts/index.html"> Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/provisioners/index.html"> Provisioners</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/gcp-dataproc.html">Google Dataproc</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/aws-emr.html">Amazon Elastic MapReduce</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/remote-hadoop.html">Remote Hadoop</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/profiles/index.html"> Profiles</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/creating-profiles.html">Creating Profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/assigning-profiles.html">Assigning Profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/admin-controls.html">Admin Controls</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/example/index.html"> Example</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../security/index.html"> 安全</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../security/client-authentication.html">Client Authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/cdap-authentication-clients-java.html">CDAP Authentication Client for Java</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/cdap-authentication-clients-python.html">CDAP Authentication Client for Python</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/custom-authentication.html">Custom Authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/authorization-extensions.html">Authorization Extensions</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../testing/index.html"> 测试和调试</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../testing/testing.html"> Testing a CDAP Application</a></li>
<li class="toctree-l2"><a class="reference internal" href="../testing/debugging.html"> Debugging</a></li>
<li class="toctree-l2"><a class="reference internal" href="../testing/troubleshooting.html"> Troubleshooting</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../ingesting-tools/index.html"> 数据融合</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-java.html">CDAP Stream Client for Java</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-python.html">CDAP Stream Client for Python</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-ruby.html">CDAP Stream Client for Ruby</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-flume.html">CDAP Flume</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../data-exploration/index.html"> 数据探索</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/filesets.html"> Fileset Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/tables.html"> Table Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/object-mapped-tables.html"> ObjectMappedTable Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/custom-datasets.html"> Custom Dataset Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/hive-execution-engines.html"> Hive Execution Engines</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/index.html"> 高级主题</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../advanced/application-logback.html"> Application Logback</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/best-practices.html"> Best Practices</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/class-loading.html"> Class Loading</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/configuring-resources.html"> Configuring Program Resources</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/program-retry-policies.html"> Program Retry Policies</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="mapreduce-programs">
<span id="mapreduce"></span><h1>MapReduce Programs<a class="headerlink" href="#mapreduce-programs" title="Permalink to this headline">🔗</a></h1>
<p>A <em>MapReduce program</em> is used to process data in batch. MapReduce can be
written as in a conventional Hadoop system. Additionally, CDAP
<a class="reference internal" href="datasets/index.html#datasets-index"><span class="std std-ref">datasets</span></a> can be accessed from MapReduce as both input and
output.</p>
<p>To process data using MapReduce, either specify <code class="docutils literal notranslate"><span class="pre">addMapReduce()</span></code> in your
application specification:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kt">void</span> <span class="nf">configure</span><span class="p">()</span> <span class="p">{</span>
  <span class="p">...</span>
  <span class="n">addMapReduce</span><span class="p">(</span><span class="k">new</span> <span class="n">WordCountJob</span><span class="p">());</span>
</pre></div>
</div>
<p>or specify <code class="docutils literal notranslate"><span class="pre">addWorkflow()</span></code> in your application and specify your MapReduce in the
<a class="reference internal" href="workflows.html#workflows"><span class="std std-ref">workflow</span></a> definition:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kt">void</span> <span class="nf">configure</span><span class="p">()</span> <span class="p">{</span>
  <span class="p">...</span>
  <span class="c1">// Run a MapReduce on the acquired data using a workflow</span>
  <span class="n">addWorkflow</span><span class="p">(</span><span class="k">new</span> <span class="n">PurchaseHistoryWorkflow</span><span class="p">());</span>
</pre></div>
</div>
<p>It is recommended to implement the <code class="docutils literal notranslate"><span class="pre">MapReduce</span></code> interface by extending the <code class="docutils literal notranslate"><span class="pre">AbstractMapReduce</span></code>
class, which allows the overriding of these three methods:</p>
<ul class="simple">
<li><code class="docutils literal notranslate"><span class="pre">configure()</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">initialize()</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">destroy()</span></code></li>
</ul>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">PurchaseHistoryBuilder</span> <span class="kd">extends</span> <span class="n">AbstractMapReduce</span> <span class="p">{</span>
  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">configure</span><span class="p">()</span> <span class="p">{</span>
    <span class="n">setName</span><span class="p">(</span><span class="s">&quot;Purchase History Builder MapReduce&quot;</span><span class="p">);</span>
    <span class="n">setDescription</span><span class="p">(</span><span class="s">&quot;Builds a purchase history for each customer&quot;</span><span class="p">);</span>
  <span class="p">}</span>
</pre></div>
</div>
<p>The configure method is similar to the one found in applications.
It defines the name and description of the MapReduce program.
You can also <a class="reference internal" href="#mapreduce-resources"><span class="std std-ref">specify resources</span></a> (memory and virtual cores)
used by the mappers and reducers.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">initialize()</span></code> method is invoked at runtime, before the MapReduce is executed.
Through the <code class="docutils literal notranslate"><span class="pre">getContext()</span></code> method you can obtain an instance of the <code class="docutils literal notranslate"><span class="pre">MapReduceContext</span></code>.
It allows you to <a class="reference internal" href="#mapreduce-datasets"><span class="std std-ref">specify datasets</span></a> to be used as input or output;
it also provides you access to the actual Hadoop job configuration, as though you were running the
MapReduce directly on Hadoop. For example, you can specify the input and output datasets,
the mapper and reducer classes as well as the intermediate data format:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="nd">@Override</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">initialize</span><span class="p">()</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
  <span class="n">MapReduceContext</span> <span class="n">context</span> <span class="o">=</span> <span class="n">getContext</span><span class="p">();</span>
  <span class="n">context</span><span class="p">.</span><span class="na">addInput</span><span class="p">(</span><span class="n">Input</span><span class="p">.</span><span class="na">ofDataset</span><span class="p">(</span><span class="s">&quot;purchases&quot;</span><span class="p">));</span>
  <span class="n">context</span><span class="p">.</span><span class="na">addOutput</span><span class="p">(</span><span class="n">Output</span><span class="p">.</span><span class="na">ofDataset</span><span class="p">(</span><span class="s">&quot;history&quot;</span><span class="p">));</span>

  <span class="n">Job</span> <span class="n">job</span> <span class="o">=</span> <span class="n">context</span><span class="p">.</span><span class="na">getHadoopJob</span><span class="p">();</span>
  <span class="n">job</span><span class="p">.</span><span class="na">setMapperClass</span><span class="p">(</span><span class="n">PurchaseMapper</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>
  <span class="n">job</span><span class="p">.</span><span class="na">setMapOutputKeyClass</span><span class="p">(</span><span class="n">Text</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>
  <span class="n">job</span><span class="p">.</span><span class="na">setMapOutputValueClass</span><span class="p">(</span><span class="n">Text</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>
  <span class="n">job</span><span class="p">.</span><span class="na">setReducerClass</span><span class="p">(</span><span class="n">PerUserReducer</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>
<span class="p">}</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">destroy()</span></code> method is invoked after the MapReduce has
finished. You could perform cleanup or send a notification of job
completion, if that was required. Through the <code class="docutils literal notranslate"><span class="pre">MapReduceContext</span></code>,
you can determine whether the MapReduce succeeded:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="nd">@Override</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">destroy</span><span class="p">()</span> <span class="p">{</span>
  <span class="n">ProgramState</span> <span class="n">state</span> <span class="o">=</span> <span class="n">getContext</span><span class="p">().</span><span class="na">getState</span><span class="p">();</span>
  <span class="c1">// do something depending on the state</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Because many MapReduce programs do not
need this method, the <code class="docutils literal notranslate"><span class="pre">AbstractMapReduce</span></code> class provides a default
implementation that does nothing.</p>
<p>CDAP <code class="docutils literal notranslate"><span class="pre">Mapper</span></code> and <code class="docutils literal notranslate"><span class="pre">Reducer</span></code> implement <a class="reference external" href="http://hadoop.apache.org/docs/r2.3.0/api/org/apache/hadoop/mapreduce/package-summary.html">the standard Hadoop APIs</a>
(note that CDAP only supports the “new” API in <code class="docutils literal notranslate"><span class="pre">org.apache.hadoop.mapreduce</span></code>, and not the
“old” API in <code class="docutils literal notranslate"><span class="pre">org.apache.hadoop.mapred</span></code>):</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">PurchaseMapper</span>
    <span class="kd">extends</span> <span class="n">Mapper</span><span class="o">&lt;</span><span class="kt">byte</span><span class="o">[]</span><span class="p">,</span> <span class="n">Purchase</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">Text</span><span class="o">&gt;</span> <span class="p">{</span>

  <span class="kd">private</span> <span class="n">Metrics</span> <span class="n">mapMetrics</span><span class="p">;</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">map</span><span class="p">(</span><span class="kt">byte</span><span class="o">[]</span> <span class="n">key</span><span class="p">,</span> <span class="n">Purchase</span> <span class="n">purchase</span><span class="p">,</span> <span class="n">Context</span> <span class="n">context</span><span class="p">)</span>
    <span class="kd">throws</span> <span class="n">IOException</span><span class="p">,</span> <span class="n">InterruptedException</span> <span class="p">{</span>
    <span class="n">String</span> <span class="n">user</span> <span class="o">=</span> <span class="n">purchase</span><span class="p">.</span><span class="na">getCustomer</span><span class="p">();</span>
    <span class="k">if</span> <span class="p">(</span><span class="n">purchase</span><span class="p">.</span><span class="na">getPrice</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mi">100000</span><span class="p">)</span> <span class="p">{</span>
      <span class="n">mapMetrics</span><span class="p">.</span><span class="na">count</span><span class="p">(</span><span class="s">&quot;purchases.large&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">);</span>
    <span class="p">}</span>
    <span class="n">context</span><span class="p">.</span><span class="na">write</span><span class="p">(</span><span class="k">new</span> <span class="n">Text</span><span class="p">(</span><span class="n">user</span><span class="p">),</span> <span class="k">new</span> <span class="n">Text</span><span class="p">(</span><span class="k">new</span> <span class="n">Gson</span><span class="p">().</span><span class="na">toJson</span><span class="p">(</span><span class="n">purchase</span><span class="p">)));</span>
  <span class="p">}</span>
<span class="p">}</span>

<span class="kd">public</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">PerUserReducer</span>
    <span class="kd">extends</span> <span class="n">Reducer</span><span class="o">&lt;</span><span class="n">Text</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">String</span><span class="p">,</span> <span class="n">PurchaseHistory</span><span class="o">&gt;</span> <span class="p">{</span>

  <span class="nd">@UseDataSet</span><span class="p">(</span><span class="s">&quot;frequentCustomers&quot;</span><span class="p">)</span>
  <span class="kd">private</span> <span class="n">KeyValueTable</span> <span class="n">frequentCustomers</span><span class="p">;</span>
  <span class="kd">private</span> <span class="n">Metrics</span> <span class="n">reduceMetrics</span><span class="p">;</span>

  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">reduce</span><span class="p">(</span><span class="n">Text</span> <span class="n">customer</span><span class="p">,</span> <span class="n">Iterable</span><span class="o">&lt;</span><span class="n">Text</span><span class="o">&gt;</span> <span class="n">values</span><span class="p">,</span> <span class="n">Context</span> <span class="n">context</span><span class="p">)</span>
    <span class="kd">throws</span> <span class="n">IOException</span><span class="p">,</span> <span class="n">InterruptedException</span> <span class="p">{</span>
    <span class="n">PurchaseHistory</span> <span class="n">purchases</span> <span class="o">=</span> <span class="k">new</span> <span class="n">PurchaseHistory</span><span class="p">(</span><span class="n">customer</span><span class="p">.</span><span class="na">toString</span><span class="p">());</span>
    <span class="kt">int</span> <span class="n">numPurchases</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
    <span class="k">for</span> <span class="p">(</span><span class="n">Text</span> <span class="n">val</span> <span class="p">:</span> <span class="n">values</span><span class="p">)</span> <span class="p">{</span>
      <span class="n">purchases</span><span class="p">.</span><span class="na">add</span><span class="p">(</span><span class="k">new</span> <span class="n">Gson</span><span class="p">().</span><span class="na">fromJson</span><span class="p">(</span><span class="n">val</span><span class="p">.</span><span class="na">toString</span><span class="p">(),</span> <span class="n">Purchase</span><span class="p">.</span><span class="na">class</span><span class="p">));</span>
      <span class="n">numPurchases</span><span class="o">++</span><span class="p">;</span>
    <span class="p">}</span>
    <span class="k">if</span> <span class="p">(</span><span class="n">numPurchases</span> <span class="o">==</span> <span class="mi">1</span><span class="p">)</span> <span class="p">{</span>
      <span class="n">reduceMetrics</span><span class="p">.</span><span class="na">count</span><span class="p">(</span><span class="s">&quot;customers.rare&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">);</span>
    <span class="p">}</span> <span class="k">else</span> <span class="k">if</span> <span class="p">(</span><span class="n">numPurchases</span> <span class="o">&gt;</span> <span class="mi">10</span><span class="p">)</span> <span class="p">{</span>
      <span class="n">reduceMetrics</span><span class="p">.</span><span class="na">count</span><span class="p">(</span><span class="s">&quot;customers.frequent&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">);</span>
      <span class="n">frequentCustomers</span><span class="p">.</span><span class="na">write</span><span class="p">(</span><span class="n">customer</span><span class="p">.</span><span class="na">toString</span><span class="p">(),</span> <span class="n">String</span><span class="p">.</span><span class="na">valueOf</span><span class="p">(</span><span class="n">numPurchases</span><span class="p">));</span>
    <span class="p">}</span>
    <span class="n">context</span><span class="p">.</span><span class="na">write</span><span class="p">(</span><span class="n">customer</span><span class="p">.</span><span class="na">toString</span><span class="p">(),</span> <span class="n">purchases</span><span class="p">);</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<div class="section" id="mapreduce-and-datasets">
<span id="mapreduce-datasets"></span><h2>MapReduce and Datasets<a class="headerlink" href="#mapreduce-and-datasets" title="Permalink to this headline">🔗</a></h2>
<p>Both a CDAP <code class="docutils literal notranslate"><span class="pre">Mapper</span></code> and <code class="docutils literal notranslate"><span class="pre">Reducer</span></code> can directly read or write to a dataset, using
one of the following options. (Note that the second and third options can be used for a
<code class="docutils literal notranslate"><span class="pre">Partitioner</span></code> or <code class="docutils literal notranslate"><span class="pre">Comparator</span></code>, if configured on the MapReduce job.)</p>
<ol class="arabic">
<li><p class="first">Inject the dataset into the mapper or reducer that uses it. This method
is useful if the name of the dataset is constant or known at compile time.
For example, to have access to a dataset named <em>catalog</em>:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">CatalogJoinMapper</span> <span class="kd">extends</span> <span class="n">Mapper</span><span class="o">&lt;</span><span class="kt">byte</span><span class="o">[]</span><span class="p">,</span> <span class="n">Purchase</span><span class="p">,</span> <span class="p">...</span><span class="o">&gt;</span> <span class="p">{</span>

  <span class="nd">@UseDataSet</span><span class="p">(</span><span class="s">&quot;catalog&quot;</span><span class="p">)</span>
  <span class="kd">private</span> <span class="n">ProductCatalog</span> <span class="n">catalog</span><span class="p">;</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">map</span><span class="p">(</span><span class="kt">byte</span><span class="o">[]</span> <span class="n">key</span><span class="p">,</span> <span class="n">Purchase</span> <span class="n">purchase</span><span class="p">,</span> <span class="n">Context</span> <span class="n">context</span><span class="p">)</span>
      <span class="kd">throws</span> <span class="n">IOException</span><span class="p">,</span> <span class="n">InterruptedException</span> <span class="p">{</span>
    <span class="c1">// join with catalog by product ID</span>
    <span class="n">Product</span> <span class="n">product</span> <span class="o">=</span> <span class="n">catalog</span><span class="p">.</span><span class="na">read</span><span class="p">(</span><span class="n">purchase</span><span class="p">.</span><span class="na">getProductId</span><span class="p">());</span>
    <span class="p">...</span>
  <span class="p">}</span>
</pre></div>
</div>
</li>
<li><p class="first">Acquire the dataset in the mapper’s or reducer’s <code class="docutils literal notranslate"><span class="pre">initialize()</span></code> method. As opposed
to the previous method, this does not require the dataset name to be constant; it
only needs to be known at the time the task starts (for example, through configuration).
Note that this requires that the mapper or reducer class implements the <code class="docutils literal notranslate"><span class="pre">ProgramLifecycle</span></code>
interface, which includes the two methods <code class="docutils literal notranslate"><span class="pre">initialize()</span></code> and <code class="docutils literal notranslate"><span class="pre">destroy()</span></code>:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">CatalogJoinMapper</span> <span class="kd">extends</span> <span class="n">Mapper</span><span class="o">&lt;</span><span class="kt">byte</span><span class="o">[]</span><span class="p">,</span> <span class="n">Purchase</span><span class="p">,</span> <span class="p">...</span><span class="o">&gt;</span>
  <span class="kd">implements</span> <span class="n">ProgramLifecycle</span><span class="o">&lt;</span><span class="n">MapReduceTaskContext</span><span class="o">&gt;</span> <span class="p">{</span>

  <span class="kd">private</span> <span class="n">ProductCatalog</span> <span class="n">catalog</span><span class="p">;</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">initialize</span><span class="p">(</span><span class="n">MapReduceTaskContext</span> <span class="n">mapReduceTaskContext</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
    <span class="n">catalog</span> <span class="o">=</span> <span class="n">mapReduceTaskContext</span><span class="p">.</span><span class="na">getDataset</span><span class="p">(</span>
        <span class="n">mapReduceTaskContext</span><span class="p">.</span><span class="na">getRuntimeArguments</span><span class="p">().</span><span class="na">get</span><span class="p">(</span><span class="s">&quot;catalog.table.name&quot;</span><span class="p">));</span>
  <span class="p">}</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">destroy</span><span class="p">()</span> <span class="p">{</span>
  <span class="p">}</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">map</span><span class="p">(</span><span class="kt">byte</span><span class="o">[]</span> <span class="n">key</span><span class="p">,</span> <span class="n">Purchase</span> <span class="n">purchase</span><span class="p">,</span> <span class="n">Context</span> <span class="n">context</span><span class="p">)</span>
      <span class="kd">throws</span> <span class="n">IOException</span><span class="p">,</span> <span class="n">InterruptedException</span> <span class="p">{</span>
    <span class="c1">// join with catalog by product ID</span>
    <span class="n">Product</span> <span class="n">product</span> <span class="o">=</span> <span class="n">catalog</span><span class="p">.</span><span class="na">read</span><span class="p">(</span><span class="n">purchase</span><span class="p">.</span><span class="na">getProductId</span><span class="p">());</span>
    <span class="p">...</span>
  <span class="p">}</span>
</pre></div>
</div>
</li>
<li><p class="first">Dynamically acquire the dataset in the mapper every time it is accessed. This is useful if
the name of the dataset is not known at initialization time; for example, if it depends on
the data passed to each <code class="docutils literal notranslate"><span class="pre">map()</span></code> call. In this case, you also implement the <code class="docutils literal notranslate"><span class="pre">ProgramLifecycle</span></code>
interface, to save the <code class="docutils literal notranslate"><span class="pre">MapReduceTaskContext</span></code> for use in the <code class="docutils literal notranslate"><span class="pre">map()</span></code> method. For example:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">CatalogJoinMapper</span> <span class="kd">extends</span> <span class="n">Mapper</span><span class="o">&lt;</span><span class="kt">byte</span><span class="o">[]</span><span class="p">,</span> <span class="n">Purchase</span><span class="p">,</span> <span class="p">...</span><span class="o">&gt;</span>
  <span class="kd">implements</span> <span class="n">ProgramLifecycle</span><span class="o">&lt;</span><span class="n">MapReduceTaskContext</span><span class="o">&gt;</span> <span class="p">{</span>

  <span class="kd">private</span> <span class="n">MapReduceTaskContext</span> <span class="n">taskContext</span><span class="p">;</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">initialize</span><span class="p">(</span><span class="n">MapReduceTaskContext</span> <span class="n">mapReduceTaskContext</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
    <span class="n">taskContext</span> <span class="o">=</span> <span class="n">mapReduceTaskContext</span><span class="p">;</span>
  <span class="p">}</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">destroy</span><span class="p">()</span> <span class="p">{</span>
  <span class="p">}</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">map</span><span class="p">(</span><span class="kt">byte</span><span class="o">[]</span> <span class="n">key</span><span class="p">,</span> <span class="n">Purchase</span> <span class="n">purchase</span><span class="p">,</span> <span class="n">Context</span> <span class="n">context</span><span class="p">)</span>
      <span class="kd">throws</span> <span class="n">IOException</span><span class="p">,</span> <span class="n">InterruptedException</span> <span class="p">{</span>
    <span class="c1">// join with catalog by product ID</span>
    <span class="n">String</span> <span class="n">catalogName</span> <span class="o">=</span> <span class="n">determineCatalogName</span><span class="p">(</span><span class="n">purchase</span><span class="p">.</span><span class="na">getProductCategory</span><span class="p">());</span>
    <span class="n">ProductCatalog</span> <span class="n">catalog</span> <span class="o">=</span> <span class="n">taskContext</span><span class="p">.</span><span class="na">getDataset</span><span class="p">(</span><span class="n">catalogName</span><span class="p">);</span>
    <span class="n">Product</span> <span class="n">product</span> <span class="o">=</span> <span class="n">catalog</span><span class="p">.</span><span class="na">read</span><span class="p">(</span><span class="n">purchase</span><span class="p">.</span><span class="na">getProductId</span><span class="p">());</span>
    <span class="p">...</span>
  <span class="p">}</span>

  <span class="kd">private</span> <span class="n">String</span> <span class="nf">determineCatalogName</span><span class="p">(</span><span class="n">String</span> <span class="n">productCategory</span><span class="p">)</span> <span class="p">{</span>
    <span class="p">...</span>
  <span class="p">}</span>
</pre></div>
</div>
</li>
</ol>
<p>See also the section on <a class="reference internal" href="datasets/overview.html#datasets-in-programs"><span class="std std-ref">Using Datasets in Programs</span></a>.</p>
<p>A MapReduce program can interact with a dataset by using it as
<a class="reference internal" href="#mapreduce-datasets-input"><span class="std std-ref">an input</span></a> or <a class="reference internal" href="#mapreduce-datasets-output"><span class="std std-ref">an output</span></a>.
The dataset needs to implement specific interfaces to support this, as described in the
following sections.</p>
<p class="rubric" id="mapreduce-datasets-input">A Dataset as the Input Source of a MapReduce Program</p>
<p>When you run a MapReduce program, you can configure it to read its input from a dataset. The
source dataset must implement the <code class="docutils literal notranslate"><span class="pre">BatchReadable</span></code> interface, which requires two methods:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">BatchReadable</span><span class="o">&lt;</span><span class="n">KEY</span><span class="p">,</span> <span class="n">VALUE</span><span class="o">&gt;</span> <span class="p">{</span>
  <span class="n">List</span><span class="o">&lt;</span><span class="n">Split</span><span class="o">&gt;</span> <span class="nf">getSplits</span><span class="p">();</span>
  <span class="n">SplitReader</span><span class="o">&lt;</span><span class="n">KEY</span><span class="p">,</span> <span class="n">VALUE</span><span class="o">&gt;</span> <span class="nf">createSplitReader</span><span class="p">(</span><span class="n">Split</span> <span class="n">split</span><span class="p">);</span>
<span class="p">}</span>
</pre></div>
</div>
<p>These two methods complement each other: <code class="docutils literal notranslate"><span class="pre">getSplits()</span></code> must return all splits of the dataset
that the MapReduce program will read; <code class="docutils literal notranslate"><span class="pre">createSplitReader()</span></code> is then called in every Mapper to
read one of the splits. Note that the <code class="docutils literal notranslate"><span class="pre">KEY</span></code> and <code class="docutils literal notranslate"><span class="pre">VALUE</span></code> type parameters of the split reader
must match the input key and value type parameters of the Mapper.</p>
<p>Because <code class="docutils literal notranslate"><span class="pre">getSplits()</span></code> has no arguments, it will typically create splits that cover the
entire dataset. If you want to use a custom selection of the input data, define another
method in your dataset with additional parameters and explicitly set the input in the
<code class="docutils literal notranslate"><span class="pre">initialize()</span></code> method.</p>
<p>For example, the system dataset <code class="docutils literal notranslate"><span class="pre">KeyValueTable</span></code> implements <code class="docutils literal notranslate"><span class="pre">BatchReadable&lt;byte[],</span> <span class="pre">byte[]&gt;</span></code>
with an extra method that allows specification of the number of splits and a range of keys:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">KeyValueTable</span> <span class="kd">extends</span> <span class="n">AbstractDataset</span>
                           <span class="kd">implements</span> <span class="n">BatchReadable</span><span class="o">&lt;</span><span class="kt">byte</span><span class="o">[]</span><span class="p">,</span> <span class="kt">byte</span><span class="o">[]&gt;</span> <span class="p">{</span>
  <span class="p">...</span>
  <span class="kd">public</span> <span class="n">List</span><span class="o">&lt;</span><span class="n">Split</span><span class="o">&gt;</span> <span class="nf">getSplits</span><span class="p">(</span><span class="kt">int</span> <span class="n">numSplits</span><span class="p">,</span> <span class="kt">byte</span><span class="o">[]</span> <span class="n">start</span><span class="p">,</span> <span class="kt">byte</span><span class="o">[]</span> <span class="n">stop</span><span class="p">);</span>
<span class="p">}</span>
</pre></div>
</div>
<p>To read a range of keys and give a hint that you want 16 splits, write:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="nd">@UseDataSet</span><span class="p">(</span><span class="s">&quot;myTable&quot;</span><span class="p">)</span>
<span class="n">KeyValueTable</span> <span class="n">kvTable</span><span class="p">;</span>
<span class="p">...</span>
<span class="nd">@Override</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">initialize</span><span class="p">()</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
  <span class="n">MapReduceContext</span> <span class="n">context</span> <span class="o">=</span> <span class="n">getContext</span><span class="p">();</span>
  <span class="p">...</span>
  <span class="n">context</span><span class="p">.</span><span class="na">addInput</span><span class="p">(</span><span class="n">Input</span><span class="p">.</span><span class="na">ofDataset</span><span class="p">(</span><span class="s">&quot;myTable&quot;</span><span class="p">,</span> <span class="n">kvTable</span><span class="p">.</span><span class="na">getSplits</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="n">startKey</span><span class="p">,</span> <span class="n">stopKey</span><span class="p">)));</span>
<span class="p">}</span>
</pre></div>
</div>
<p class="rubric" id="mapreduce-datasets-output">A Dataset as the Output Destination of a MapReduce Program</p>
<p>Just as you have the option to read input from a dataset, you have the option to write to a dataset as
the output destination of a MapReduce program if that dataset implements the <code class="docutils literal notranslate"><span class="pre">BatchWritable</span></code>
interface:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">BatchWritable</span><span class="o">&lt;</span><span class="n">KEY</span><span class="p">,</span> <span class="n">VALUE</span><span class="o">&gt;</span> <span class="p">{</span>
  <span class="kt">void</span> <span class="nf">write</span><span class="p">(</span><span class="n">KEY</span> <span class="n">key</span><span class="p">,</span> <span class="n">VALUE</span> <span class="n">value</span><span class="p">);</span>
<span class="p">}</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">write()</span></code> method is used to redirect all writes performed by a Reducer to the dataset.
Again, the <code class="docutils literal notranslate"><span class="pre">KEY</span></code> and <code class="docutils literal notranslate"><span class="pre">VALUE</span></code> type parameters must match the output key and value type
parameters of the Reducer.</p>
<p class="rubric">Multiple Output Destinations of a MapReduce Program</p>
<p>To write to multiple output datasets from a MapReduce program, begin by adding the datasets as outputs:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kt">void</span> <span class="nf">initialize</span><span class="p">()</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
  <span class="p">...</span>
  <span class="n">getContext</span><span class="p">().</span><span class="na">addOutput</span><span class="p">(</span><span class="n">Output</span><span class="p">.</span><span class="na">ofDataset</span><span class="p">(</span><span class="s">&quot;productCounts&quot;</span><span class="p">));</span>
  <span class="n">getContext</span><span class="p">().</span><span class="na">addOutput</span><span class="p">(</span><span class="n">Output</span><span class="p">.</span><span class="na">ofDataset</span><span class="p">(</span><span class="s">&quot;catalog&quot;</span><span class="p">));</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Then, have the <code class="docutils literal notranslate"><span class="pre">Mapper</span></code> and/or <code class="docutils literal notranslate"><span class="pre">Reducer</span></code> implement <code class="docutils literal notranslate"><span class="pre">ProgramLifeCycle&lt;MapReduceTaskContext&gt;</span></code>.
This is to obtain access to the <code class="docutils literal notranslate"><span class="pre">MapReduceTaskContext</span></code> in their initialization methods and
to be able to write using the write method of the <code class="docutils literal notranslate"><span class="pre">MapReduceTaskContext</span></code>:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">CustomMapper</span> <span class="kd">extends</span> <span class="n">Mapper</span><span class="o">&lt;</span><span class="n">LongWritable</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">NullWritable</span><span class="p">,</span> <span class="n">Text</span><span class="o">&gt;</span>
  <span class="kd">implements</span> <span class="n">ProgramLifecycle</span><span class="o">&lt;</span><span class="n">MapReduceTaskContext</span><span class="o">&lt;</span><span class="n">NullWritable</span><span class="p">,</span> <span class="n">Text</span><span class="o">&gt;&gt;</span> <span class="p">{</span>

  <span class="kd">private</span> <span class="n">MapReduceTaskContext</span><span class="o">&lt;</span><span class="n">NullWritable</span><span class="p">,</span> <span class="n">Text</span><span class="o">&gt;</span> <span class="n">mapReduceTaskContext</span><span class="p">;</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">initialize</span><span class="p">(</span><span class="n">MapReduceTaskContext</span><span class="o">&lt;</span><span class="n">NullWritable</span><span class="p">,</span> <span class="n">Text</span><span class="o">&gt;</span> <span class="n">context</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
    <span class="k">this</span><span class="p">.</span><span class="na">mapReduceTaskContext</span> <span class="o">=</span> <span class="n">context</span><span class="p">;</span>
  <span class="p">}</span>

  <span class="kd">protected</span> <span class="kt">void</span> <span class="nf">map</span><span class="p">(</span><span class="n">LongWritable</span> <span class="n">key</span><span class="p">,</span> <span class="n">Text</span> <span class="n">value</span><span class="p">,</span> <span class="n">Context</span> <span class="n">context</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">IOException</span><span class="p">,</span> <span class="n">InterruptedException</span> <span class="p">{</span>
    <span class="c1">// compute some condition</span>
    <span class="p">...</span>
    <span class="k">if</span> <span class="p">(</span><span class="n">someCondition</span><span class="p">)</span> <span class="p">{</span>
      <span class="n">mapReduceTaskContext</span><span class="p">.</span><span class="na">write</span><span class="p">(</span><span class="s">&quot;productCounts&quot;</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">);</span>
    <span class="p">}</span> <span class="k">else</span> <span class="p">{</span>
      <span class="n">mapReduceTaskContext</span><span class="p">.</span><span class="na">write</span><span class="p">(</span><span class="s">&quot;catalog&quot;</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">);</span>
    <span class="p">}</span>
  <span class="p">}</span>

<span class="p">}</span>
</pre></div>
</div>
<p>Note that the multiple output write method—<code class="docutils literal notranslate"><span class="pre">MapReduceTaskContext.write(String,</span> <span class="pre">KEY</span> <span class="pre">key,</span> <span class="pre">VALUE</span> <span class="pre">value)</span></code>—can
only be used if there are multiple outputs. Similarly, the single output write
method—<code class="docutils literal notranslate"><span class="pre">MapReduceTaskContext.write(KEY</span> <span class="pre">key,</span> <span class="pre">VALUE</span> <span class="pre">value)</span></code>—can only be used if there
is a single output to the MapReduce program.</p>
</div>
<div class="section" id="mapreduce-and-transactions">
<span id="mapreduce-transactions"></span><h2>MapReduce and Transactions<a class="headerlink" href="#mapreduce-and-transactions" title="Permalink to this headline">🔗</a></h2>
<p>When you run a MapReduce that interacts with datasets, the system creates a
long-running transaction. Similar to the transaction of a service handler, here are
some rules to follow:</p>
<ul class="simple">
<li>Reads can only see the writes of other transactions that were committed
at the time the long-running transaction was started.</li>
<li>All writes of the long-running transaction are committed atomically,
and only become visible to other transactions after they are committed.</li>
<li>Tasks of a MapReduce program may not see each other’s writes because of buffering of
datasets.</li>
</ul>
<p>However, there is a key difference: long-running transactions do not participate in
conflict detection. If another transaction overlaps with the long-running transaction and
writes to the same row, it will not cause a conflict but simply overwrite it.</p>
<p>It is not efficient to fail the long-running job based on a single conflict. Because of
this, it is not recommended to write to the same dataset from both real-time and MapReduce
programs. It is better to use different datasets, or at least ensure that the real-time
processing writes to a disjoint set of columns.</p>
<p>It’s important to note that the MapReduce framework will reattempt a task (Mapper or
Reducer) if it fails. If the task is writing to a dataset, the reattempt of the task will
most likely repeat the writes that were already performed in the failed attempt. Therefore,
it is highly advisable that all writes performed by MapReduce programs be idempotent.</p>
<p>If your Map Reduce Job performs speculative execution, a similar repetition of
of writes can occur.</p>
<p>See <a class="reference internal" href="transaction-system.html#transaction-system"><span class="std std-ref">Transaction System</span></a> for additional information.</p>
</div>
<div class="section" id="mapreduce-and-resources">
<span id="mapreduce-resources"></span><h2>MapReduce and Resources<a class="headerlink" href="#mapreduce-and-resources" title="Permalink to this headline">🔗</a></h2>
<p>Both the YARN container size and the number of virtual cores used in a MapReduce job can be specified
as part of the configuration. They can also be set at runtime through the use of runtime arguments.</p>
<p>The Resources API, if called with two arguments, sets both the memory used in megabytes
and the number of virtual cores used.</p>
</div>
<div class="section" id="mapreduce-program-examples">
<h2>MapReduce Program Examples<a class="headerlink" href="#mapreduce-program-examples" title="Permalink to this headline">🔗</a></h2>
<ul class="simple">
<li>For an example, the how-to guide <span class="xref std std-ref">cdap-mapreduce-guide</span> demonstrates the use of MapReduce.</li>
</ul>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Cube Dataset" href="datasets/cube.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="Plugins" href="plugins.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../_static/js/js.cookie.js"></script><script type="text/javascript" src="../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>